#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2018-09-05 17:04:33
# Project: imdb
import re
from pyspider.libs.base_handler import *


class Handler(BaseHandler):
    crawl_config = {
    }

    @every(minutes=24 * 60)
    def on_start(self):
        self.crawl('http://www.imdb.com/search/title?count=100&title_type=feature,tv_series,tv_movie&ref_=nv_ch_mm_1',
                   callback=self.index_page, validate_cert=False)

    @config(age=10 * 24 * 60 * 60)
    def index_page(self, response):
        for each in response.doc('a[href^="http"]').items():
            if re.match("https://www.imdb.com/title/tt\d+/\?ref_=adv_li_i", each.attr.href):
                self.crawl(each.attr.href, callback=self.detail_page, validate_cert=False)
            self.crawl([x.attr.href for x in response.doc('.next-page').items()], callback=self.index_page,
                       validate_cert=False)

    @config(priority=2)
    def detail_page(self, response):
        participants = [x.text() for x in response.doc('.credit_summary_item').items()]
        creators = [e.split('\n')[1] for e in participants if e.split('\n')[0] == 'Creators:']
        cretors = creators[0].split(',')
        return {
            "url": response.url,
            "title": response.doc('h1').text(),
            "rating": response.doc('strong > span').text(),
            "director": cretors,
        }
